SherbyNet - /page/wikibook/entry.mu

<!DOCTYPE html>

<title>Multinomial logistic regression</title>

</head>

Multinomial logistic regression

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><div role="note" class="hatnote navigation-not-searchable">"Multinomial regression" redirects here. For the related Probit procedure, see <a href="Multinomial_probit" title="Multinomial probit">Multinomial probit</a>.</div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .ambox{border:1px solid #a2a9b1;border-left:10px solid #36c;background-color:#fbfbfb;box-sizing:border-box}.mw-parser-output .ambox+link+.ambox,.mw-parser-output .ambox+link+style+.ambox,.mw-parser-output .ambox+link+link+.ambox,.mw-parser-output .ambox+.mw-empty-elt+link+.ambox,.mw-parser-output .ambox+.mw-empty-elt+link+style+.ambox,.mw-parser-output .ambox+.mw-empty-elt+link+link+.ambox{margin-top:-1px}html body.mediawiki .mw-parser-output .ambox.mbox-small-left{margin:4px 1em 4px 0;overflow:hidden;width:238px;border-collapse:collapse;font-size:88%;line-height:1.25em}.mw-parser-output .ambox-speedy{border-left:10px solid #b32424;background-color:#fee7e6}.mw-parser-output .ambox-delete{border-left:10px solid #b32424}.mw-parser-output .ambox-content{border-left:10px solid #f28500}.mw-parser-output .ambox-style{border-left:10px solid #fc3}.mw-parser-output .ambox-move{border-left:10px solid #9932cc}.mw-parser-output .ambox-protection{border-left:10px solid #a2a9b1}.mw-parser-output .ambox .mbox-text{border:none;padding:0.25em 0.5em;width:100%}.mw-parser-output .ambox .mbox-image{border:none;padding:2px 0 2px 0.5em;text-align:center}.mw-parser-output .ambox .mbox-imageright{border:none;padding:2px 0.5em 2px 0;text-align:center}.mw-parser-output .ambox .mbox-empty-cell{border:none;padding:0;width:1px}.mw-parser-output .ambox .mbox-image-div{width:52px}@media(min-width:720px){.mw-parser-output .ambox{margin:0 10%}}@media print{body.ns-0 .mw-parser-output .ambox{display:none!important}}

/* end https://en.wikipedia.org/ */

</style>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar nomobile nowraplinks hlist"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Regression_analysis" title="Regression analysis">Regression analysis</a></th></tr><tr><th class="sidebar-heading">

Models</th></tr><tr><td class="sidebar-content">

<ul><li><a href="Linear_regression" title="Linear regression">Linear regression</a></li>

<li><a href="Simple_linear_regression" title="Simple linear regression">Simple regression</a></li>

<li><a href="Polynomial_regression" title="Polynomial regression">Polynomial regression</a></li>

<li><a href="General_linear_model" title="General linear model">General linear model</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Generalized_linear_model" title="Generalized linear model">Generalized linear model</a></li>

<li><a href="Vector_generalized_linear_model" title="Vector generalized linear model">Vector generalized linear model</a></li>

<li><a href="Discrete_choice" title="Discrete choice">Discrete choice</a></li>

<li><a href="Binomial_regression" title="Binomial regression">Binomial regression</a></li>

<li><a href="Binary_regression" title="Binary regression">Binary regression</a></li>

<li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Mixed_logit" title="Mixed logit">Mixed logit</a></li>

<li><a href="Probit_model" title="Probit model">Probit</a></li>

<li><a href="Multinomial_probit" title="Multinomial probit">Multinomial probit</a></li>

<li><a href="Ordered_logit" title="Ordered logit">Ordered logit</a></li>

<li><a href="Ordered_probit" class="mw-redirect" title="Ordered probit">Ordered probit</a></li>

<li><a href="Poisson_regression" title="Poisson regression">Poisson</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Multilevel_model" title="Multilevel model">Multilevel model</a></li>

<li><a href="Fixed_effects_model" title="Fixed effects model">Fixed effects</a></li>

<li><a href="Random_effects_model" title="Random effects model">Random effects</a></li>

<li><a href="Mixed_model" title="Mixed model">Linear mixed-effects model</a></li>

<li><a href="Nonlinear_mixed-effects_model" title="Nonlinear mixed-effects model">Nonlinear mixed-effects model</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Nonlinear_regression" title="Nonlinear regression">Nonlinear regression</a></li>

<li><a href="Nonparametric_regression" title="Nonparametric regression">Nonparametric</a></li>

<li><a href="Semiparametric_regression" title="Semiparametric regression">Semiparametric</a></li>

<li><a href="Robust_regression" title="Robust regression">Robust</a></li>

<li><a href="Quantile_regression" title="Quantile regression">Quantile</a></li>

<li><a href="Isotonic_regression" title="Isotonic regression">Isotonic</a></li>

<li><a href="Principal_component_regression" title="Principal component regression">Principal components</a></li>

<li><a href="Least-angle_regression" title="Least-angle regression">Least angle</a></li>

<li><a href="Local_regression" title="Local regression">Local</a></li>

<li><a href="Segmented_regression" title="Segmented regression">Segmented</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Errors-in-variables_models" class="mw-redirect" title="Errors-in-variables models">Errors-in-variables</a></li></ul></td>

</tr><tr><th class="sidebar-heading">

Estimation</th></tr><tr><td class="sidebar-content">

<ul><li><a href="Least_squares" title="Least squares">Least squares</a></li>

<li><a href="Linear_least_squares" title="Linear least squares">Linear</a></li>

<li><a href="Non-linear_least_squares" title="Non-linear least squares">Non-linear</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Ordinary_least_squares" title="Ordinary least squares">Ordinary</a></li>

<li><a href="Weighted_least_squares" title="Weighted least squares">Weighted</a></li>

<li><a href="Generalized_least_squares" title="Generalized least squares">Generalized</a></li>

<li><a href="Generalized_estimating_equation" title="Generalized estimating equation">Generalized estimating equation</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Partial_least_squares_regression" title="Partial least squares regression">Partial</a></li>

<li><a href="Total_least_squares" title="Total least squares">Total</a></li>

<li><a href="Non-negative_least_squares" title="Non-negative least squares">Non-negative</a></li>

<li><a href="Tikhonov_regularization" class="mw-redirect" title="Tikhonov regularization">Ridge regression</a></li>

<li><a href="Regularized_least_squares" title="Regularized least squares">Regularized</a></li></ul></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Least_absolute_deviations" title="Least absolute deviations">Least absolute deviations</a></li>

<li><a href="Iteratively_reweighted_least_squares" title="Iteratively reweighted least squares">Iteratively reweighted</a></li>

<li><a href="Bayesian_linear_regression" title="Bayesian linear regression">Bayesian</a></li>

<li><a href="Bayesian_multivariate_linear_regression" title="Bayesian multivariate linear regression">Bayesian multivariate</a></li>

<li><a href="Least-squares_spectral_analysis" title="Least-squares spectral analysis">Least-squares spectral analysis</a></li></ul></td>

</tr><tr><th class="sidebar-heading">

Background</th></tr><tr><td class="sidebar-content">

<ul><li><a href="Regression_validation" title="Regression validation">Regression validation</a></li>

<li><a href="Mean_and_predicted_response" class="mw-redirect" title="Mean and predicted response">Mean and predicted response</a></li>

<li><a href="Errors_and_residuals" title="Errors and residuals">Errors and residuals</a></li>

<li><a href="Goodness_of_fit" title="Goodness of fit">Goodness of fit</a></li>

<li><a href="Studentized_residual" title="Studentized residual">Studentized residual</a></li>

<li><a href="Gauss%E2%80%93Markov_theorem" title="Gauss–Markov theorem">Gauss–Markov theorem</a></li></ul></td>

</tr><tr><td class="sidebar-below">

<ul><li> <a href="Portal%3AMathematics" title="Portal:Mathematics">Mathematics portal</a></li></ul></td></tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

In <a href="Statistics" title="Statistics">statistics</a>, multinomial logistic regression is a <a href="Statistical_classification" title="Statistical classification">classification</a> method that generalizes <a href="Logistic_regression" title="Logistic regression">logistic regression</a> to <a href="Multiclass_classification" title="Multiclass classification">multiclass problems</a>, i.e. with more than two possible discrete outcomes.<a href="#cite_note-1">[1]</a> That is, it is a model that is used to predict the probabilities of the different possible outcomes of a <a href="Categorical_distribution" title="Categorical distribution">categorically distributed</a> <a href="Dependent_variable" class="mw-redirect" title="Dependent variable">dependent variable</a>, given a set of <a href="Independent_variable" class="mw-redirect" title="Independent variable">independent variables</a> (which may be real-valued, binary-valued, categorical-valued, etc.).

Multinomial logistic regression is known by a variety of other names, including polytomous LR,<a href="#cite_note-2">[2]</a><a href="#cite_note-3">[3]</a> multiclass LR, <a href="Softmax_activation_function" class="mw-redirect" title="Softmax activation function">softmax</a> regression, multinomial logit (mlogit), the maximum entropy (MaxEnt) classifier, and the conditional maximum entropy model.<a href="#cite_note-malouf-4">[4]</a>

<div class="mw-heading mw-heading2"><h2 id="Background">Background</h2></div>

Multinomial logistic regression is used when the <a href="Dependent_variable" class="mw-redirect" title="Dependent variable">dependent variable</a> in question is <a href="Level_of_measurement#Nominal_measurement" title="Level of measurement">nominal</a> (equivalently categorical, meaning that it falls into any one of a set of categories that cannot be ordered in any meaningful way) and for which there are more than two categories. Some examples would be:

<ul><li>Which major will a college student choose, given their grades, stated likes and dislikes, etc.?</li>

<li>Which blood type does a person have, given the results of various diagnostic tests?</li>

<li>In a hands-free mobile phone dialing application, which person's name was spoken, given various properties of the speech signal?</li>

<li>Which candidate will a person vote for, given particular demographic characteristics?</li>

<li>Which country will a firm locate an office in, given the characteristics of the firm and of the various candidate countries?</li></ul>

These are all <a href="Statistical_classification" title="Statistical classification">statistical classification</a> problems. They all have in common a <a href="Dependent_variable" class="mw-redirect" title="Dependent variable">dependent variable</a> to be predicted that comes from one of a limited set of items that cannot be meaningfully ordered, as well as a set of <a href="Independent_variable" class="mw-redirect" title="Independent variable">independent variables</a> (also known as features, explanators, etc.), which are used to predict the dependent variable. Multinomial logistic regression is a particular solution to classification problems that use a linear combination of the observed features and some problem-specific parameters to estimate the probability of each particular value of the dependent variable. The best values of the parameters for a given problem are usually determined from some training data (e.g. some people for whom both the diagnostic test results and blood types are known, or some examples of known words being spoken).

<div class="mw-heading mw-heading2"><h2 id="Assumptions">Assumptions</h2></div>

The multinomial logistic model assumes that data are case-specific; that is, each independent variable has a single value for each case. As with other types of regression, there is no need for the independent variables to be <a href="Statistically_independent" class="mw-redirect" title="Statistically independent">statistically independent</a> from each other (unlike, for example, in a <a href="Naive_Bayes_classifier" title="Naive Bayes classifier">naive Bayes classifier</a>); however, <a href="Multicollinearity" title="Multicollinearity">collinearity</a> is assumed to be relatively low, as it becomes difficult to differentiate between the impact of several variables if this is not the case.<a href="#cite_note-5">[5]</a>

If the multinomial logit is used to model choices, it relies on the assumption of <a href="Independence_of_irrelevant_alternatives" title="Independence of irrelevant alternatives">independence of irrelevant alternatives</a> (IIA), which is not always desirable. This assumption states that the odds of preferring one class over another do not depend on the presence or absence of other "irrelevant" alternatives. For example, the relative probabilities of taking a car or bus to work do not change if a bicycle is added as an additional possibility. This allows the choice of K alternatives to be modeled as a set of K − 1 independent binary choices, in which one alternative is chosen as a "pivot" and the other K − 1 compared against it, one at a time. The IIA hypothesis is a core hypothesis in rational choice theory; however numerous studies in psychology show that individuals often violate this assumption when making choices. An example of a problem case arises if choices include a car and a blue bus. Suppose the odds ratio between the two is 1 : 1. Now if the option of a red bus is introduced, a person may be indifferent between a red and a blue bus, and hence may exhibit a car : blue bus : red bus odds ratio of 1 : 0.5 : 0.5, thus maintaining a 1 : 1 ratio of car : any bus while adopting a changed car : blue bus ratio of 1 : 0.5. Here the red bus option was not in fact irrelevant, because a red bus was a <a href="Perfect_substitute" class="mw-redirect" title="Perfect substitute">perfect substitute</a> for a blue bus.

If the multinomial logit is used to model choices, it may in some situations impose too much constraint on the relative preferences between the different alternatives. It is especially important to take into account if the analysis aims to predict how choices would change if one alternative were to disappear (for instance if one political candidate withdraws from a three candidate race). Other models like the <a href="Nested_logit" class="mw-redirect" title="Nested logit">nested logit</a> or the <a href="Multinomial_probit" title="Multinomial probit">multinomial probit</a> may be used in such cases as they allow for violation of the IIA.<a href="#cite_note-6">[6]</a>

<div class="mw-heading mw-heading2"><h2 id="Model">Model</h2></div>

<div role="note" class="hatnote navigation-not-searchable">See also: <a href="Logistic_regression" title="Logistic regression">Logistic regression</a></div>

<div class="mw-heading mw-heading3"><h3 id="Introduction">Introduction</h3></div>

There are multiple equivalent ways to describe the mathematical model underlying multinomial logistic regression. This can make it difficult to compare different treatments of the subject in different texts. The article on <a href="Logistic_regression" title="Logistic regression">logistic regression</a> presents a number of equivalent formulations of simple logistic regression, and many of these have analogues in the multinomial logit model.

The idea behind all of them, as in many other <a href="Statistical_classification" title="Statistical classification">statistical classification</a> techniques, is to construct a <a href="Linear_predictor_function" title="Linear predictor function">linear predictor function</a> that constructs a score from a set of weights that are <a href="Linear_combination" title="Linear combination">linearly combined</a> with the explanatory variables (features) of a given observation using a <a href="Dot_product" title="Dot product">dot product</a>:

<mi>score</mi>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {score} (\mathbf {X} _{i},k)={\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i},}</annotation>

</semantics>

</math><img src="./d7810d95ad9b2bfbe7384432e83fa311ace76b03.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.795ex; height:2.843ex;" alt="{\displaystyle \operatorname {score} (\mathbf {X} _{i},k)={\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i},}" loading="lazy"></dd></dl>

where Xi is the vector of explanatory variables describing observation i, βk is a vector of weights (or <a href="Regression_coefficient" class="mw-redirect" title="Regression coefficient">regression coefficients</a>) corresponding to outcome k, and score(Xi, k) is the score associated with assigning observation i to category k. In <a href="Discrete_choice" title="Discrete choice">discrete choice</a> theory, where observations represent people and outcomes represent choices, the score is considered the <a href="Utility" title="Utility">utility</a> associated with person i choosing outcome k. The predicted outcome is the one with the highest score.

The difference between the multinomial logit model and numerous other methods, models, algorithms, etc. with the same basic setup (the <a href="Perceptron" title="Perceptron">perceptron</a> algorithm, <a href="Support_vector_machine" title="Support vector machine">support vector machines</a>, <a href="Linear_discriminant_analysis" title="Linear discriminant analysis">linear discriminant analysis</a>, etc.) is the procedure for determining (training) the optimal weights/coefficients and the way that the score is interpreted. In particular, in the multinomial logit model, the score can directly be converted to a probability value, indicating the <a href="Probability" title="Probability">probability</a> of observation i choosing outcome k given the measured characteristics of the observation. This provides a principled way of incorporating the prediction of a particular multinomial logit model into a larger procedure that may involve multiple such predictions, each with a possibility of error. Without such means of combining predictions, errors tend to multiply. For example, imagine a large <a href="Predictive_modelling" title="Predictive modelling">predictive model</a> that is broken down into a series of submodels where the prediction of a given submodel is used as the input of another submodel, and that prediction is in turn used as the input into a third submodel, etc. If each submodel has 90% accuracy in its predictions, and there are five submodels in series, then the overall model has only 0.95 = 59% accuracy. If each submodel has 80% accuracy, then overall accuracy drops to 0.85 = 33% accuracy. This issue is known as <a href="Error_propagation" class="mw-redirect" title="Error propagation">error propagation</a> and is a serious problem in real-world predictive models, which are usually composed of numerous parts. Predicting probabilities of each possible outcome, rather than simply making a single optimal prediction, is one means of alleviating this issue.

<div class="mw-heading mw-heading3"><h3 id="Setup">Setup</h3></div>

The basic setup is the same as in <a href="Logistic_regression" title="Logistic regression">logistic regression</a>, the only difference being that the <a href="Dependent_variable" class="mw-redirect" title="Dependent variable">dependent variables</a> are <a href="Categorical_variable" title="Categorical variable">categorical</a> rather than <a href="Binary_variable" class="mw-redirect" title="Binary variable">binary</a>, i.e. there are K possible outcomes rather than just two. The following description is somewhat shortened; for more details, consult the <a href="Logistic_regression" title="Logistic regression">logistic regression</a> article.

<div class="mw-heading mw-heading4"><h4 id="Data_points">Data points</h4></div>

Specifically, it is assumed that we have a series of N observed data points. Each data point i (ranging from 1 to N) consists of a set of M explanatory variables x1,i ... xM,i (also known as <a href="Independent_variable" class="mw-redirect" title="Independent variable">independent variables</a>, predictor variables, features, etc.), and an associated <a href="Categorical_variable" title="Categorical variable">categorical</a> outcome Yi (also known as <a href="Dependent_variable" class="mw-redirect" title="Dependent variable">dependent variable</a>, response variable), which can take on one of K possible values. These possible values represent logically separate categories (e.g. different political parties, blood types, etc.), and are often described mathematically by arbitrarily assigning each a number from 1 to K. The explanatory variables and outcome represent observed properties of the data points, and are often thought of as originating in the observations of N "experiments" — although an "experiment" may consist of nothing more than gathering data. The goal of multinomial logistic regression is to construct a model that explains the relationship between the explanatory variables and the outcome, so that the outcome of a new "experiment" can be correctly predicted for a new data point for which the explanatory variables, but not the outcome, are available. In the process, the model attempts to explain the relative effect of differing explanatory variables on the outcome.

Some examples:

<ul><li>The observed outcomes are different variants of a disease such as <a href="Hepatitis" title="Hepatitis">hepatitis</a> (possibly including "no disease" and/or other related diseases) in a set of patients, and the explanatory variables might be characteristics of the patients thought to be pertinent (sex, race, age, <a href="Blood_pressure" title="Blood pressure">blood pressure</a>, outcomes of various liver-function tests, etc.). The goal is then to predict which disease is causing the observed liver-related symptoms in a new patient.</li>

<li>The observed outcomes are the party chosen by a set of people in an election, and the explanatory variables are the demographic characteristics of each person (e.g. sex, race, age, income, etc.). The goal is then to predict the likely vote of a new voter with given characteristics.</li></ul>

<div class="mw-heading mw-heading4"><h4 id="Linear_predictor">Linear predictor</h4></div>

As in other forms of linear regression, multinomial logistic regression uses a <a href="Linear_predictor_function" title="Linear predictor function">linear predictor function</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle f(k,i)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f(k,i)}</annotation>

</semantics>

</math><img src="./811817b0a554c4823cf74b54ec0a28fbbcc44756.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.136ex; height:2.843ex;" alt="{\displaystyle f(k,i)}" loading="lazy"> to predict the probability that observation i has outcome k, of the following form:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f(k,i)=\beta _{0,k}+\beta _{1,k}x_{1,i}+\beta _{2,k}x_{2,i}+\cdots +\beta _{M,k}x_{M,i},}</annotation>

</semantics>

</math><img src="./2cb74185a381069c19dfc9ceb09c34b0a887ac8a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:50.737ex; height:3.009ex;" alt="{\displaystyle f(k,i)=\beta _{0,k}+\beta _{1,k}x_{1,i}+\beta _{2,k}x_{2,i}+\cdots +\beta _{M,k}x_{M,i},}" loading="lazy"></dd></dl>

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta _{m,k}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta _{m,k}}</annotation>

</semantics>

</math><img src="./b4dcbd15fb2f28d1babf127afabe8ea103e8b457.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:4.305ex; height:2.843ex;" alt="{\displaystyle \beta _{m,k}}" loading="lazy"> is a <a href="Regression_coefficient" class="mw-redirect" title="Regression coefficient">regression coefficient</a> associated with the mth explanatory variable and the kth outcome. As explained in the <a href="Logistic_regression" title="Logistic regression">logistic regression</a> article, the regression coefficients and explanatory variables are normally grouped into vectors of size M + 1, so that the predictor function can be written more compactly:

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f(k,i)={\boldsymbol {\beta }}_{k}\cdot \mathbf {x} _{i},}</annotation>

</semantics>

</math><img src="./eb11a10791be260ba57f1e213e4d6afc79390674.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:16.393ex; height:2.843ex;" alt="{\displaystyle f(k,i)={\boldsymbol {\beta }}_{k}\cdot \mathbf {x} _{i},}" loading="lazy"></dd></dl>

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\boldsymbol {\beta }}_{k}}">

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\boldsymbol {\beta }}_{k}}</annotation>

</semantics>

</math><img src="./07f1bcb6340535f54accaf88124f4cd5ba1d4fc8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.623ex; height:2.676ex;" alt="{\displaystyle {\boldsymbol {\beta }}_{k}}" loading="lazy"> is the set of regression coefficients associated with outcome k, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {x} _{i}}">

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {x} _{i}}</annotation>

</semantics>

</math><img src="./57d2ef3df60acdb53bdf90535264041fea7231cd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.211ex; height:2.009ex;" alt="{\displaystyle \mathbf {x} _{i}}" loading="lazy"> (a row vector) is the set of explanatory variables associated with observation i, prepended by a 1 in entry 0.

<div class="mw-heading mw-heading3"><h3 id="As_a_set_of_independent_binary_regressions">As a set of independent binary regressions</h3></div>

To arrive at the multinomial logit model, one can imagine, for K possible outcomes, running K independent binary logistic regression models, in which one outcome is chosen as a "pivot" and then the other K − 1 outcomes are separately regressed against the pivot outcome. If outcome K (the last outcome) is chosen as the pivot, the K − 1 regression equations are:

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \ln {\frac {\Pr(Y_{i}=k)}{\Pr(Y_{i}=K)}}\,=\,{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i},\;\;\;\;\;\;1\leq k<K}</annotation>

</semantics>

</math><img src="./70cdde1873f5fbe834ea45e83d168e54c12192d0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.671ex; width:41.316ex; height:6.509ex;" alt="{\displaystyle \ln {\frac {\Pr(Y_{i}=k)}{\Pr(Y_{i}=K)}}\,=\,{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i},\;\;\;\;\;\;1\leq k<K}" loading="lazy">.</dd></dl>

This formulation is also known as the <a href="Compositional_data#Additive_log_ratio_transform" title="Compositional data">Additive Log Ratio</a> transform commonly used in compositional data analysis. In other applications it’s referred to as “relative risk”.<a href="#cite_note-7">[7]</a>

If we exponentiate both sides and solve for the probabilities, we get:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)\,=\,{\Pr(Y_{i}=K)}\;e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}},\;\;\;\;\;\;1\leq k<K}</annotation>

</semantics>

</math><img src="./b24ad33ccd1649cc06bfa38c6e4350e79c25214d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:48.212ex; height:3.176ex;" alt="{\displaystyle \Pr(Y_{i}=k)\,=\,{\Pr(Y_{i}=K)}\;e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}},\;\;\;\;\;\;1\leq k<K}" loading="lazy"></dd></dl>

Using the fact that all K of the probabilities must sum to one, we find:

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mrow>

</mtd>

<mtd>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

</mrow>

</mtd>

<mtd>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mrow>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

</mrow>

</mtd>

<mtd>

<mfrac>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\Pr(Y_{i}=K)={}&1-\sum _{j=1}^{K-1}\Pr(Y_{i}=j)\\={}&1-\sum _{j=1}^{K-1}{\Pr(Y_{i}=K)}\;e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}\;\;\Rightarrow \;\;\Pr(Y_{i}=K)\\={}&{\frac {1}{1+\sum _{j=1}^{K-1}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}}.\end{aligned}}}</annotation>

</semantics>

</math><img src="./d9d1b5b239dafc937d2f18145d1b2adf30c5039d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -10.671ex; width:59.121ex; height:22.509ex;" alt="{\displaystyle {\begin{aligned}\Pr(Y_{i}=K)={}&1-\sum _{j=1}^{K-1}\Pr(Y_{i}=j)\\={}&1-\sum _{j=1}^{K-1}{\Pr(Y_{i}=K)}\;e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}\;\;\Rightarrow \;\;\Pr(Y_{i}=K)\\={}&{\frac {1}{1+\sum _{j=1}^{K-1}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}}.\end{aligned}}}" loading="lazy"></dd></dl>

We can use this to find the other probabilities:

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{1+\sum _{j=1}^{K-1}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}},\;\;\;\;\;\;1\leq k<K}</annotation>

</semantics>

</math><img src="./a468beb77dc39e5b212d930a4db673fc0ffd1c25.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:46.502ex; height:7.343ex;" alt="{\displaystyle \Pr(Y_{i}=k)={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{1+\sum _{j=1}^{K-1}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}},\;\;\;\;\;\;1\leq k<K}" loading="lazy">.</dd></dl>

The fact that we run multiple regressions reveals why the model relies on the assumption of <a href="Independence_of_irrelevant_alternatives" title="Independence of irrelevant alternatives">independence of irrelevant alternatives</a> described above.

<div class="mw-heading mw-heading3"><h3 id="Estimating_the_coefficients">Estimating the coefficients</h3></div>

The unknown parameters in each vector βk are typically jointly estimated by <a href="Maximum_a_posteriori" class="mw-redirect" title="Maximum a posteriori">maximum a posteriori</a> (MAP) estimation, which is an extension of <a href="Maximum_likelihood" class="mw-redirect" title="Maximum likelihood">maximum likelihood</a> using <a href="Regularization_(mathematics)" title="Regularization (mathematics)">regularization</a> of the weights to prevent pathological solutions (usually a squared regularizing function, which is equivalent to placing a zero-mean <a href="Gaussian_distribution" class="mw-redirect" title="Gaussian distribution">Gaussian</a> <a href="Prior_distribution" class="mw-redirect" title="Prior distribution">prior distribution</a> on the weights, but other distributions are also possible). The solution is typically found using an iterative procedure such as <a href="Generalized_iterative_scaling" title="Generalized iterative scaling">generalized iterative scaling</a>,<a href="#cite_note-8">[8]</a> <a href="Iteratively_reweighted_least_squares" title="Iteratively reweighted least squares">iteratively reweighted least squares</a> (IRLS),<a href="#cite_note-9">[9]</a> by means of <a href="Gradient-based_optimization" class="mw-redirect" title="Gradient-based optimization">gradient-based optimization</a> algorithms such as <a href="L-BFGS" class="mw-redirect" title="L-BFGS">L-BFGS</a>,<a href="#cite_note-malouf-4">[4]</a> or by specialized <a href="Coordinate_descent" title="Coordinate descent">coordinate descent</a> algorithms.<a href="#cite_note-10">[10]</a>

<div class="mw-heading mw-heading3"><h3 id="As_a_log-linear_model">As a log-linear model</h3></div>

The formulation of binary logistic regression as a <a href="Logistic_regression#log-linear_model" title="Logistic regression">log-linear model</a> can be directly extended to multi-way regression. That is, we model the <a href="Logarithm" title="Logarithm">logarithm</a> of the probability of seeing a given output using the linear predictor as well as an additional <a href="Normalization_factor" class="mw-redirect" title="Normalization factor">normalization factor</a>, the logarithm of the <a href="Partition_function_(mathematics)" title="Partition function (mathematics)">partition function</a>:

<msub>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \ln \Pr(Y_{i}=k)={\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}-\ln Z,\;\;\;\;\;\;1\leq k\leq K.}</annotation>

</semantics>

</math><img src="./4c63052d00b4b5350b1d9e1ec37c8a47891046ce.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:46.345ex; height:2.843ex;" alt="{\displaystyle \ln \Pr(Y_{i}=k)={\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}-\ln Z,\;\;\;\;\;\;1\leq k\leq K.}" loading="lazy"></dd></dl>

As in the binary case, we need an extra term <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle -\ln Z}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle -\ln Z}</annotation>

</semantics>

</math><img src="./bfc4529e0b901532d643b75ee89a2235538218ed.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:6.202ex; height:2.343ex;" alt="{\displaystyle -\ln Z}" loading="lazy"> to ensure that the whole set of probabilities forms a <a href="Probability_distribution" title="Probability distribution">probability distribution</a>, i.e. so that they all sum to one:

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sum _{k=1}^{K}\Pr(Y_{i}=k)=1}</annotation>

</semantics>

</math><img src="./e43de307488bca7f86f15d1e2b67c3d6ef065582.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:18.767ex; height:7.343ex;" alt="{\displaystyle \sum _{k=1}^{K}\Pr(Y_{i}=k)=1}" loading="lazy"></dd></dl>

The reason why we need to add a term to ensure normalization, rather than multiply as is usual, is because we have taken the logarithm of the probabilities. Exponentiating both sides turns the additive term into a multiplicative factor, so that the probability is just the <a href="Gibbs_measure" title="Gibbs measure">Gibbs measure</a>:

<msub>

</mrow>

</msub>

<mfrac>

</mfrac>

</mrow>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)={\frac {1}{Z}}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}},\;\;\;\;\;\;1\leq k\leq K.}</annotation>

</semantics>

</math><img src="./d499caf2089b06ea60726adaa3d89430cdad0059.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:38.337ex; height:5.176ex;" alt="{\displaystyle \Pr(Y_{i}=k)={\frac {1}{Z}}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}},\;\;\;\;\;\;1\leq k\leq K.}" loading="lazy"></dd></dl>

The quantity Z is called the <a href="Partition_function_(mathematics)" title="Partition function (mathematics)">partition function</a> for the distribution. We can compute the value of the partition function by applying the above constraint that requires all probabilities to sum to 1:

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</munderover>

<mfrac>

</mfrac>

</mrow>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mfrac>

</mfrac>

</mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 1=\sum _{k=1}^{K}\Pr(Y_{i}=k)\;=\;\sum _{k=1}^{K}{\frac {1}{Z}}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}\;=\;{\frac {1}{Z}}\sum _{k=1}^{K}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}.}</annotation>

</semantics>

</math><img src="./45d979a9a1dc9649763d6564e2d0cc673cd7cad2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:52.636ex; height:7.343ex;" alt="{\displaystyle 1=\sum _{k=1}^{K}\Pr(Y_{i}=k)\;=\;\sum _{k=1}^{K}{\frac {1}{Z}}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}\;=\;{\frac {1}{Z}}\sum _{k=1}^{K}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}.}" loading="lazy"></dd></dl>

Therefore

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Z=\sum _{k=1}^{K}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}.}</annotation>

</semantics>

</math><img src="./065fd74572e55b9ef143ddea2d984abaf1c6d571.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.005ex; width:14.938ex; height:7.343ex;" alt="{\displaystyle Z=\sum _{k=1}^{K}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}.}" loading="lazy"></dd></dl>

Note that this factor is "constant" in the sense that it is not a function of Yi, which is the variable over which the probability distribution is defined. However, it is definitely not constant with respect to the explanatory variables, or crucially, with respect to the unknown regression coefficients βk, which we will need to determine through some sort of <a href="Mathematical_optimization" title="Mathematical optimization">optimization</a> procedure.

The resulting equations for the probabilities are

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}},\;\;\;\;\;\;1\leq k\leq K.}</annotation>

</semantics>

</math><img src="./e8eb61deab2e13580d98c6020e7fdb33e9dea2bd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:42.363ex; height:7.343ex;" alt="{\displaystyle \Pr(Y_{i}=k)={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}},\;\;\;\;\;\;1\leq k\leq K.}" loading="lazy"></dd></dl>

The following function:

<mi>softmax</mi>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {softmax} (k,s_{1},\ldots ,s_{K})={\frac {e^{s_{k}}}{\sum _{j=1}^{K}e^{s_{j}}}}}</annotation>

</semantics>

</math><img src="./92d2b05d1c4a1ee74e64b4f7542145170445d0d2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:34.769ex; height:6.843ex;" alt="{\displaystyle \operatorname {softmax} (k,s_{1},\ldots ,s_{K})={\frac {e^{s_{k}}}{\sum _{j=1}^{K}e^{s_{j}}}}}" loading="lazy"></dd></dl>

is referred to as the <a href="Softmax_function" title="Softmax function">softmax function</a>. The reason is that the effect of exponentiating the values <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s_{1},\ldots ,s_{K}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle s_{1},\ldots ,s_{K}}</annotation>

</semantics>

</math><img src="./1c865450f02fd46f380fef15f69c183d34626a19.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:10.106ex; height:2.009ex;" alt="{\displaystyle s_{1},\ldots ,s_{K}}" loading="lazy"> is to exaggerate the differences between them. As a result, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {softmax} (k,s_{1},\ldots ,s_{K})}">

<mi>softmax</mi>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {softmax} (k,s_{1},\ldots ,s_{K})}</annotation>

</semantics>

</math><img src="./534e56fb34000b2aebb71c72a5f06cb18e86dbe4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.182ex; height:2.843ex;" alt="{\displaystyle \operatorname {softmax} (k,s_{1},\ldots ,s_{K})}" loading="lazy"> will return a value close to 0 whenever <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle s_{k}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle s_{k}}</annotation>

</semantics>

</math><img src="./04f159343172781e7666dbc88280c91f34117c30.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.179ex; height:2.009ex;" alt="{\displaystyle s_{k}}" loading="lazy"> is significantly less than the maximum of all the values, and will return a value close to 1 when applied to the maximum value, unless it is extremely close to the next-largest value. Thus, the softmax function can be used to construct a <a href="Weighted_average" title="Weighted average">weighted average</a> that behaves as a <a href="Smooth_function" class="mw-redirect" title="Smooth function">smooth function</a> (which can be conveniently <a href="Differentiation_(mathematics)" class="mw-redirect" title="Differentiation (mathematics)">differentiated</a>, etc.) and which approximates the <a href="Indicator_function" title="Indicator function">indicator function</a>

<mrow>

<mtr>

<mtd>

</mtd>

<mtd>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

</mtd>

<mtd>

<mtext>otherwise</mtext>

</mrow>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle f(k)={\begin{cases}1&{\textrm {if}}\;k=\operatorname {\arg \max } _{j}s_{j},\\0&{\textrm {otherwise}}.\end{cases}}}</annotation>

</semantics>

</math><img src="./abdec8770a48f9d2b9167ff37a9045caf786449c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:31.584ex; height:6.176ex;" alt="{\displaystyle f(k)={\begin{cases}1&{\textrm {if}}\;k=\operatorname {\arg \max } _{j}s_{j},\\0&{\textrm {otherwise}}.\end{cases}}}" loading="lazy"></dd></dl>

Thus, we can write the probability equations as

<msub>

</mrow>

</msub>

<mi>softmax</mi>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)=\operatorname {softmax} (k,{\boldsymbol {\beta }}_{1}\cdot \mathbf {X} _{i},\ldots ,{\boldsymbol {\beta }}_{K}\cdot \mathbf {X} _{i})}</annotation>

</semantics>

</math><img src="./0cae9a7e99bd0fc580fc7bddd22d1d4582c71715.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:45.928ex; height:2.843ex;" alt="{\displaystyle \Pr(Y_{i}=k)=\operatorname {softmax} (k,{\boldsymbol {\beta }}_{1}\cdot \mathbf {X} _{i},\ldots ,{\boldsymbol {\beta }}_{K}\cdot \mathbf {X} _{i})}" loading="lazy"></dd></dl>

The softmax function thus serves as the equivalent of the <a href="Logistic_function" title="Logistic function">logistic function</a> in binary logistic regression.

Note that not all of the <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\boldsymbol {\beta }}_{k}}">

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\boldsymbol {\beta }}_{k}}</annotation>

</semantics>

</math><img src="./07f1bcb6340535f54accaf88124f4cd5ba1d4fc8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:2.623ex; height:2.676ex;" alt="{\displaystyle {\boldsymbol {\beta }}_{k}}" loading="lazy"> vectors of coefficients are uniquely <a href="Identifiability" title="Identifiability">identifiable</a>. This is due to the fact that all probabilities must sum to 1, making one of them completely determined once all the rest are known. As a result, there are only <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K-1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K-1}</annotation>

</semantics>

</math><img src="./dd6e429474c269979f75b41db5d334243b3dccd3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:6.069ex; height:2.343ex;" alt="{\displaystyle K-1}" loading="lazy"> separately specifiable probabilities, and hence <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K-1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K-1}</annotation>

</semantics>

</math><img src="./dd6e429474c269979f75b41db5d334243b3dccd3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.505ex; width:6.069ex; height:2.343ex;" alt="{\displaystyle K-1}" loading="lazy"> separately identifiable vectors of coefficients. One way to see this is to note that if we add a constant vector to all of the coefficient vectors, the equations are identical:

<mtr>

<mtd>

<mfrac>

<msup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mtd>

<mtd>

<mfrac>

<mrow>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mtd>

</mtr>

<mtr>

<mtd>

<mfrac>

<mrow>

<msup>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

<mrow>

<msup>

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mtd>

</mtr>

<mtr>

<mtd>

<mfrac>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}{\frac {e^{({\boldsymbol {\beta }}_{k}+\mathbf {C} )\cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{({\boldsymbol {\beta }}_{j}+\mathbf {C} )\cdot \mathbf {X} _{i}}}}&={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}e^{\mathbf {C} \cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}e^{\mathbf {C} \cdot \mathbf {X} _{i}}}}\\&={\frac {e^{\mathbf {C} \cdot \mathbf {X} _{i}}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{e^{\mathbf {C} \cdot \mathbf {X} _{i}}\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}}\\&={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}}\end{aligned}}}</annotation>

</semantics>

</math><img src="./1abb5acc4ac5305a682d78ae51b3c150c289eca7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -10.671ex; width:37.977ex; height:22.509ex;" alt="{\displaystyle {\begin{aligned}{\frac {e^{({\boldsymbol {\beta }}_{k}+\mathbf {C} )\cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{({\boldsymbol {\beta }}_{j}+\mathbf {C} )\cdot \mathbf {X} _{i}}}}&={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}e^{\mathbf {C} \cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}e^{\mathbf {C} \cdot \mathbf {X} _{i}}}}\\&={\frac {e^{\mathbf {C} \cdot \mathbf {X} _{i}}e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{e^{\mathbf {C} \cdot \mathbf {X} _{i}}\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}}\\&={\frac {e^{{\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}}}{\sum _{j=1}^{K}e^{{\boldsymbol {\beta }}_{j}\cdot \mathbf {X} _{i}}}}\end{aligned}}}" loading="lazy"></dd></dl>

As a result, it is conventional to set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {C} =-{\boldsymbol {\beta }}_{K}}">

</mrow>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {C} =-{\boldsymbol {\beta }}_{K}}</annotation>

</semantics>

</math><img src="./345cf4cbde600926afdbc72161a6c8d05cd9a89e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:10.065ex; height:2.676ex;" alt="{\displaystyle \mathbf {C} =-{\boldsymbol {\beta }}_{K}}" loading="lazy"> (or alternatively, one of the other coefficient vectors). Essentially, we set the constant so that one of the vectors becomes <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\boldsymbol {0}}}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\boldsymbol {0}}}</annotation>

</semantics>

</math><img src="./58fe04d1d35aac19861a0d7c9f7c374c88d42db0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.337ex; height:2.176ex;" alt="{\displaystyle {\boldsymbol {0}}}" loading="lazy">, and all of the other vectors get transformed into the difference between those vectors and the vector we chose. This is equivalent to "pivoting" around one of the K choices, and examining how much better or worse all of the other K − 1 choices are, relative to the choice we are pivoting around. Mathematically, we transform the coefficients as follows:

<mtr>

<mtd>

</mrow>

</mrow>

</msubsup>

</mtd>

<mtd>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

</mrow>

</mrow>

</msubsup>

</mtd>

<mtd>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}{\boldsymbol {\beta }}'_{k}&={\boldsymbol {\beta }}_{k}-{\boldsymbol {\beta }}_{K},\;\;\;\;1\leq k<K,\\{\boldsymbol {\beta }}'_{K}&=0.\end{aligned}}}</annotation>

</semantics>

</math><img src="./db8474b3fba9afa0f82227175b8b19c0e038bdcc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:30.665ex; height:6.176ex;" alt="{\displaystyle {\begin{aligned}{\boldsymbol {\beta }}'_{k}&={\boldsymbol {\beta }}_{k}-{\boldsymbol {\beta }}_{K},\;\;\;\;1\leq k<K,\\{\boldsymbol {\beta }}'_{K}&=0.\end{aligned}}}" loading="lazy"></dd></dl>

This leads to the following equations:

<msub>

</mrow>

</msub>

<mfrac>

<msup>

</mrow>

</mrow>

</msubsup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

</mrow>

</mrow>

</msubsup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)={\frac {e^{{\boldsymbol {\beta }}'_{k}\cdot \mathbf {X} _{i}}}{1+\sum _{j=1}^{K-1}e^{{\boldsymbol {\beta }}'_{j}\cdot \mathbf {X} _{i}}}},\;\;\;\;\;\;1\leq k\leq K}</annotation>

</semantics>

</math><img src="./fc7cf433154d2d75dd1b85d14e6ccb34fc6180aa.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.671ex; width:46.502ex; height:7.676ex;" alt="{\displaystyle \Pr(Y_{i}=k)={\frac {e^{{\boldsymbol {\beta }}'_{k}\cdot \mathbf {X} _{i}}}{1+\sum _{j=1}^{K-1}e^{{\boldsymbol {\beta }}'_{j}\cdot \mathbf {X} _{i}}}},\;\;\;\;\;\;1\leq k\leq K}" loading="lazy"></dd></dl>

Other than the prime symbols on the regression coefficients, this is exactly the same as the form of the model described above, in terms of K − 1 independent two-way regressions.

<div class="mw-heading mw-heading3"><h3 id="As_a_latent-variable_model">As a latent-variable model</h3></div>

It is also possible to formulate multinomial logistic regression as a latent variable model, following the <a href="Logistic_regression#Two-way_latent-variable_model" title="Logistic regression">two-way latent variable model</a> described for binary logistic regression. This formulation is common in the theory of <a href="Discrete_choice" title="Discrete choice">discrete choice</a> models, and makes it easier to compare multinomial logistic regression to the related <a href="Multinomial_probit" title="Multinomial probit">multinomial probit</a> model, as well as to extend it to more complex models.

Imagine that, for each data point i and possible outcome k = 1,2,...,K, there is a continuous <a href="Latent_variable" class="mw-redirect" title="Latent variable">latent variable</a> Yi,k* (i.e. an unobserved <a href="Random_variable" title="Random variable">random variable</a>) that is distributed as follows:

</mrow>

</mrow>

</msubsup>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Y_{i,k}^{\ast }={\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}+\varepsilon _{k}\;\;\;\;,\;\;k\leq K}</annotation>

</semantics>

</math><img src="./1cf7613de4b63da8edb4eb0ff2b16cb5d391c148.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:29.977ex; height:3.176ex;" alt="{\displaystyle Y_{i,k}^{\ast }={\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}+\varepsilon _{k}\;\;\;\;,\;\;k\leq K}" loading="lazy"></dd></dl>

where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \varepsilon _{k}\sim \operatorname {EV} _{1}(0,1),}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \varepsilon _{k}\sim \operatorname {EV} _{1}(0,1),}</annotation>

</semantics>

</math><img src="./41de383261fa14bcd308d2390fbb28f6e3ce35e1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.466ex; height:2.843ex;" alt="{\displaystyle \varepsilon _{k}\sim \operatorname {EV} _{1}(0,1),}" loading="lazy"> i.e. a standard type-1 <a href="Extreme_value_distribution" class="mw-redirect" title="Extreme value distribution">extreme value distribution</a>.

This latent variable can be thought of as the <a href="Utility" title="Utility">utility</a> associated with data point i choosing outcome k, where there is some randomness in the actual amount of utility obtained, which accounts for other unmodeled factors that go into the choice. The value of the actual variable <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Y_{i}}</annotation>

</semantics>

</math><img src="./d57be496fff95ee2a97ee43c7f7fe244b4dbf8ae.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.15ex; height:2.509ex;" alt="{\displaystyle Y_{i}}" loading="lazy"> is then determined in a non-random fashion from these latent variables (i.e. the randomness has been moved from the observed outcomes into the latent variables), where outcome k is chosen <a href="If_and_only_if" title="If and only if">if and only if</a> the associated utility (the value of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y_{i,k}^{\ast }}">

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Y_{i,k}^{\ast }}</annotation>

</semantics>

</math><img src="./33eb7f636a5049b93f92a3eed751e0533948f07a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:3.464ex; height:3.176ex;" alt="{\displaystyle Y_{i,k}^{\ast }}" loading="lazy">) is greater than the utilities of all the other choices, i.e. if the utility associated with outcome k is the maximum of all the utilities. Since the latent variables are <a href="Continuous_variable" class="mw-redirect" title="Continuous variable">continuous</a>, the probability of two having exactly the same value is 0, so we ignore the scenario. That is:

<dl><dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}\Pr(Y_{i}=1)&=\Pr(Y_{i,1}^{\ast }>Y_{i,2}^{\ast }{\text{ and }}Y_{i,1}^{\ast }>Y_{i,3}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,1}^{\ast }>Y_{i,K}^{\ast })\\\Pr(Y_{i}=2)&=\Pr(Y_{i,2}^{\ast }>Y_{i,1}^{\ast }{\text{ and }}Y_{i,2}^{\ast }>Y_{i,3}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,2}^{\ast }>Y_{i,K}^{\ast })\\&\,\,\,\vdots \\\Pr(Y_{i}=K)&=\Pr(Y_{i,K}^{\ast }>Y_{i,1}^{\ast }{\text{ and }}Y_{i,K}^{\ast }>Y_{i,2}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,K}^{\ast }>Y_{i,K-1}^{\ast })\\\end{aligned}}}">

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mtd>

</mtr>

<mtr>

<mtd>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\Pr(Y_{i}=1)&=\Pr(Y_{i,1}^{\ast }>Y_{i,2}^{\ast }{\text{ and }}Y_{i,1}^{\ast }>Y_{i,3}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,1}^{\ast }>Y_{i,K}^{\ast })\\\Pr(Y_{i}=2)&=\Pr(Y_{i,2}^{\ast }>Y_{i,1}^{\ast }{\text{ and }}Y_{i,2}^{\ast }>Y_{i,3}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,2}^{\ast }>Y_{i,K}^{\ast })\\&\,\,\,\vdots \\\Pr(Y_{i}=K)&=\Pr(Y_{i,K}^{\ast }>Y_{i,1}^{\ast }{\text{ and }}Y_{i,K}^{\ast }>Y_{i,2}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,K}^{\ast }>Y_{i,K-1}^{\ast })\\\end{aligned}}}</annotation>

</semantics>

</math><img src="./8e280be3f36649906431730edfcf92e08823779b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -6.838ex; width:72.525ex; height:14.843ex;" alt="{\displaystyle {\begin{aligned}\Pr(Y_{i}=1)&=\Pr(Y_{i,1}^{\ast }>Y_{i,2}^{\ast }{\text{ and }}Y_{i,1}^{\ast }>Y_{i,3}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,1}^{\ast }>Y_{i,K}^{\ast })\\\Pr(Y_{i}=2)&=\Pr(Y_{i,2}^{\ast }>Y_{i,1}^{\ast }{\text{ and }}Y_{i,2}^{\ast }>Y_{i,3}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,2}^{\ast }>Y_{i,K}^{\ast })\\&\,\,\,\vdots \\\Pr(Y_{i}=K)&=\Pr(Y_{i,K}^{\ast }>Y_{i,1}^{\ast }{\text{ and }}Y_{i,K}^{\ast }>Y_{i,2}^{\ast }{\text{ and }}\cdots {\text{ and }}Y_{i,K}^{\ast }>Y_{i,K-1}^{\ast })\\\end{aligned}}}" loading="lazy"></dd></dl>

Or equivalently:

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \Pr(Y_{i}=k)\;=\;\Pr(\max(Y_{i,1}^{\ast },Y_{i,2}^{\ast },\ldots ,Y_{i,K}^{\ast })=Y_{i,k}^{\ast })\;\;\;\;,\;\;k\leq K}</annotation>

</semantics>

</math><img src="./85e9b395d4d39883caedef6ae792e26b2bc9ef53.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.338ex; width:60.574ex; height:3.343ex;" alt="{\displaystyle \Pr(Y_{i}=k)\;=\;\Pr(\max(Y_{i,1}^{\ast },Y_{i,2}^{\ast },\ldots ,Y_{i,K}^{\ast })=Y_{i,k}^{\ast })\;\;\;\;,\;\;k\leq K}" loading="lazy"></dd></dl>

Let's look more closely at the first equation, which we can write as follows:

<dl><dd><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}\Pr(Y_{i}=1)&=\Pr(Y_{i,1}^{\ast }>Y_{i,k}^{\ast }\ \forall \ k=2,\ldots ,K)\\&=\Pr(Y_{i,1}^{\ast }-Y_{i,k}^{\ast }>0\ \forall \ k=2,\ldots ,K)\\&=\Pr({\boldsymbol {\beta }}_{1}\cdot \mathbf {X} _{i}+\varepsilon _{1}-({\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}+\varepsilon _{k})>0\ \forall \ k=2,\ldots ,K)\\&=\Pr(({\boldsymbol {\beta }}_{1}-{\boldsymbol {\beta }}_{k})\cdot \mathbf {X} _{i}>\varepsilon _{k}-\varepsilon _{1}\ \forall \ k=2,\ldots ,K)\end{aligned}}}">

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mtd>

</mtr>

<mtr>

<mtd>

</mrow>

</mrow>

</msubsup>

</mrow>

</mrow>

</msubsup>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}\Pr(Y_{i}=1)&=\Pr(Y_{i,1}^{\ast }>Y_{i,k}^{\ast }\ \forall \ k=2,\ldots ,K)\\&=\Pr(Y_{i,1}^{\ast }-Y_{i,k}^{\ast }>0\ \forall \ k=2,\ldots ,K)\\&=\Pr({\boldsymbol {\beta }}_{1}\cdot \mathbf {X} _{i}+\varepsilon _{1}-({\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}+\varepsilon _{k})>0\ \forall \ k=2,\ldots ,K)\\&=\Pr(({\boldsymbol {\beta }}_{1}-{\boldsymbol {\beta }}_{k})\cdot \mathbf {X} _{i}>\varepsilon _{k}-\varepsilon _{1}\ \forall \ k=2,\ldots ,K)\end{aligned}}}</annotation>

</semantics>

</math><img src="./e960e5129f14aeb19596491b4fa8ece11c3e6149.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -6.171ex; width:67.148ex; height:13.509ex;" alt="{\displaystyle {\begin{aligned}\Pr(Y_{i}=1)&=\Pr(Y_{i,1}^{\ast }>Y_{i,k}^{\ast }\ \forall \ k=2,\ldots ,K)\\&=\Pr(Y_{i,1}^{\ast }-Y_{i,k}^{\ast }>0\ \forall \ k=2,\ldots ,K)\\&=\Pr({\boldsymbol {\beta }}_{1}\cdot \mathbf {X} _{i}+\varepsilon _{1}-({\boldsymbol {\beta }}_{k}\cdot \mathbf {X} _{i}+\varepsilon _{k})>0\ \forall \ k=2,\ldots ,K)\\&=\Pr(({\boldsymbol {\beta }}_{1}-{\boldsymbol {\beta }}_{k})\cdot \mathbf {X} _{i}>\varepsilon _{k}-\varepsilon _{1}\ \forall \ k=2,\ldots ,K)\end{aligned}}}" loading="lazy"></dd></dl>

There are a few things to realize here:

<ol><li>In general, if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X\sim \operatorname {EV} _{1}(a,b)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X\sim \operatorname {EV} _{1}(a,b)}</annotation>

</semantics>

</math><img src="./b950c0d339f29af3db351aca73db6d24f9cb6b68.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.529ex; height:2.843ex;" alt="{\displaystyle X\sim \operatorname {EV} _{1}(a,b)}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y\sim \operatorname {EV} _{1}(a,b)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Y\sim \operatorname {EV} _{1}(a,b)}</annotation>

</semantics>

</math><img src="./c1e23aae9481579541829efafdedca1839b6bb88.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.323ex; height:2.843ex;" alt="{\displaystyle Y\sim \operatorname {EV} _{1}(a,b)}" loading="lazy"> then <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X-Y\sim \operatorname {Logistic} (0,b).}">

<mi>Logistic</mi>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X-Y\sim \operatorname {Logistic} (0,b).}</annotation>

</semantics>

</math><img src="./8738abee03d0702495cce4b29b3ff76d9468e03a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:23.267ex; height:2.843ex;" alt="{\displaystyle X-Y\sim \operatorname {Logistic} (0,b).}" loading="lazy"> That is, the difference of two <a href="Independent_identically_distributed" class="mw-redirect" title="Independent identically distributed">independent identically distributed</a> extreme-value-distributed variables follows the <a href="Logistic_distribution" title="Logistic distribution">logistic distribution</a>, where the first parameter is unimportant. This is understandable since the first parameter is a <a href="Location_parameter" title="Location parameter">location parameter</a>, i.e. it shifts the mean by a fixed amount, and if two values are both shifted by the same amount, their difference remains the same. This means that all of the relational statements underlying the probability of a given choice involve the logistic distribution, which makes the initial choice of the extreme-value distribution, which seemed rather arbitrary, somewhat more understandable.</li>

<li>The second parameter in an extreme-value or logistic distribution is a <a href="Scale_parameter" title="Scale parameter">scale parameter</a>, such that if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X\sim \operatorname {Logistic} (0,1)}">

<mi>Logistic</mi>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X\sim \operatorname {Logistic} (0,1)}</annotation>

</semantics>

</math><img src="./30b3f76d845425eba3374505a198c1d1a1ccc96c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.171ex; height:2.843ex;" alt="{\displaystyle X\sim \operatorname {Logistic} (0,1)}" loading="lazy"> then <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle bX\sim \operatorname {Logistic} (0,b).}">

<mi>Logistic</mi>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle bX\sim \operatorname {Logistic} (0,b).}</annotation>

</semantics>

</math><img src="./4a45ea113ea95dfc3b5a2cd907c05a266e766b50.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.651ex; height:2.843ex;" alt="{\displaystyle bX\sim \operatorname {Logistic} (0,b).}" loading="lazy"> This means that the effect of using an error variable with an arbitrary scale parameter in place of scale 1 can be compensated simply by multiplying all regression vectors by the same scale. Together with the previous point, this shows that the use of a standard extreme-value distribution (location 0, scale 1) for the error variables entails no loss of generality over using an arbitrary extreme-value distribution. In fact, the model is <a href="Nonidentifiable" class="mw-redirect" title="Nonidentifiable">nonidentifiable</a> (no single set of optimal coefficients) if the more general distribution is used.</li>

<li>Because only differences of vectors of regression coefficients are used, adding an arbitrary constant to all coefficient vectors has no effect on the model. This means that, just as in the log-linear model, only K − 1 of the coefficient vectors are identifiable, and the last one can be set to an arbitrary value (e.g. 0).</li></ol>

Actually finding the values of the above probabilities is somewhat difficult, and is a problem of computing a particular <a href="Order_statistic" title="Order statistic">order statistic</a> (the first, i.e. maximum) of a set of values. However, it can be shown that the resulting expressions are the same as in above formulations, i.e. the two are equivalent.

<div class="mw-heading mw-heading2"><h2 id="Estimation_of_intercept">Estimation of intercept</h2></div>

When using multinomial logistic regression, one category of the dependent variable is chosen as the reference category. Separate <a href="Odds_ratio" title="Odds ratio">odds ratios</a> are determined for all independent variables for each category of the dependent variable with the exception of the reference category, which is omitted from the analysis. The exponential beta coefficient represents the change in the odds of the dependent variable being in a particular category vis-a-vis the reference category, associated with a one unit change of the corresponding independent variable.

<div class="mw-heading mw-heading2"><h2 id="Likelihood_function">Likelihood function</h2></div>

The observed values <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{i}\in \{1,\dots ,K\}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y_{i}\in \{1,\dots ,K\}}</annotation>

</semantics>

</math><img src="./ff0defa670b42083fb95a36b895a2a2e2613c04c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.511ex; height:2.843ex;" alt="{\displaystyle y_{i}\in \{1,\dots ,K\}}" loading="lazy"> for <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i=1,\dots ,n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i=1,\dots ,n}</annotation>

</semantics>

</math><img src="./f3f269b2f3b2f87fec0168426652a5ea80b56112.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.636ex; height:2.509ex;" alt="{\displaystyle i=1,\dots ,n}" loading="lazy"> of the explained variables are considered as realizations of stochastically independent, <a href="Categorical_distribution" title="Categorical distribution">categorically distributed</a> random variables <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle Y_{1},\dots ,Y_{n}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Y_{1},\dots ,Y_{n}}</annotation>

</semantics>

</math><img src="./010c105dda336a4624a635ea54886fb040034d64.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:10.152ex; height:2.509ex;" alt="{\displaystyle Y_{1},\dots ,Y_{n}}" loading="lazy">.

The <a href="Likelihood_function" title="Likelihood function">likelihood function</a> for this model is defined by

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</munderover>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

<msup>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L=\prod _{i=1}^{n}P(Y_{i}=y_{i})=\prod _{i=1}^{n}\prod _{j=1}^{K}P(Y_{i}=j)^{\delta _{j,y_{i}}},}</annotation>

</semantics>

</math><img src="./4aeb064328b9d49bb2a6ba35486bed10fb4a178b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.338ex; width:42.296ex; height:7.676ex;" alt="{\displaystyle L=\prod _{i=1}^{n}P(Y_{i}=y_{i})=\prod _{i=1}^{n}\prod _{j=1}^{K}P(Y_{i}=j)^{\delta _{j,y_{i}}},}" loading="lazy"></dd></dl>

where the index <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy"> denotes the observations 1 to n and the index <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle j}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle j}</annotation>

</semantics>

</math><img src="./2f461e54f5c093e92a55547b9764291390f0b5d0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; margin-left: -0.027ex; width:0.985ex; height:2.509ex;" alt="{\displaystyle j}" loading="lazy"> denotes the classes 1 to K. <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \delta _{j,y_{i}}={\begin{cases}1,{\text{ for }}j=y_{i}\\0,{\text{ otherwise}}\end{cases}}}">

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<mrow>

<mtr>

<mtd>

</mrow>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

<mtext> otherwise</mtext>

</mrow>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \delta _{j,y_{i}}={\begin{cases}1,{\text{ for }}j=y_{i}\\0,{\text{ otherwise}}\end{cases}}}</annotation>

</semantics>

</math><img src="./43c1083e09653bacea524a526c7c37dfe963e6fa.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:21.777ex; height:6.176ex;" alt="{\displaystyle \delta _{j,y_{i}}={\begin{cases}1,{\text{ for }}j=y_{i}\\0,{\text{ otherwise}}\end{cases}}}" loading="lazy"> is the <a href="Kronecker_delta" title="Kronecker delta">Kronecker delta</a>.

The negative log-likelihood function is therefore the well-known cross-entropy:

</mrow>

</mrow>

</munderover>

</mrow>

</mrow>

</munderover>

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mrow>

</munder>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle -\log L=-\sum _{i=1}^{n}\sum _{j=1}^{K}\delta _{j,y_{i}}\log(P(Y_{i}=j))=-\sum _{j=1}^{K}\sum _{y_{i}=j}\log(P(Y_{i}=j)).}</annotation>

</semantics>

</math><img src="./ecdc0f26f2870df9c165eaa100fd0425f6511c8c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -3.505ex; width:66.673ex; height:7.843ex;" alt="{\displaystyle -\log L=-\sum _{i=1}^{n}\sum _{j=1}^{K}\delta _{j,y_{i}}\log(P(Y_{i}=j))=-\sum _{j=1}^{K}\sum _{y_{i}=j}\log(P(Y_{i}=j)).}" loading="lazy"></dd></dl>

<div class="mw-heading mw-heading2"><h2 id="Application_in_natural_language_processing">Application in natural language processing</h2></div>

In <a href="Natural_language_processing" title="Natural language processing">natural language processing</a>, multinomial LR classifiers are commonly used as an alternative to <a href="Naive_Bayes_classifier" title="Naive Bayes classifier">naive Bayes classifiers</a> because they do not assume <a href="Statistical_independence" class="mw-redirect" title="Statistical independence">statistical independence</a> of the random variables (commonly known as features) that serve as predictors. However, learning in such a model is slower than for a naive Bayes classifier, and thus may not be appropriate given a very large number of classes to learn. In particular, learning in a naive Bayes classifier is a simple matter of counting up the number of co-occurrences of features and classes, while in a maximum entropy classifier the weights, which are typically maximized using <a href="Maximum_a_posteriori" class="mw-redirect" title="Maximum a posteriori">maximum a posteriori</a> (MAP) estimation, must be learned using an iterative procedure; see <a href="#Estimating_the_coefficients">#Estimating the coefficients</a>.

<ul><li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Multinomial_probit" title="Multinomial probit">Multinomial probit</a></li></ul>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em;">

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFGreene2012" class="citation book cs1"><a href="William_Greene_(economist)" title="William Greene (economist)">Greene, William H.</a> (2012). Econometric Analysis (Seventh ed.). Boston: Pearson Education. pp. 803–806. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-273-75356-8</bdi>.</cite>

</li>

<li id="cite_note-2"><a href="#cite_ref-2">^</a> <cite id="CITEREFEngel1988" class="citation journal cs1">Engel, J. (1988). "Polytomous logistic regression". Statistica Neerlandica. 42 (4): 233–252. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1111%2Fj.1467-9574.1988.tb01238.x">10.1111/j.1467-9574.1988.tb01238.x</a>.</cite>

</li>

<li id="cite_note-3"><a href="#cite_ref-3">^</a> <cite id="CITEREFMenard2002" class="citation book cs1">Menard, Scott (2002). <a rel="nofollow" class="external text" href="https://archive.org/details/appliedlogisticr00mena">Applied Logistic Regression Analysis</a>. SAGE. p. <a rel="nofollow" class="external text" href="https://archive.org/details/appliedlogisticr00mena/page/n99">91</a>. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>9780761922087</bdi>.</cite>

</li>

<li id="cite_note-malouf-4">^ <a href="#cite_ref-malouf_4-0">a</a> <a href="#cite_ref-malouf_4-1">b</a> <cite id="CITEREFMalouf2002" class="citation conference cs1">Malouf, Robert (2002). <a rel="nofollow" class="external text" href="http://aclweb.org/anthology/W/W02/W02-2018.pdf">A comparison of algorithms for maximum entropy parameter estimation</a> (PDF). Sixth Conf. on Natural Language Learning (CoNLL). pp. 49–55.</cite>

</li>

<li id="cite_note-5"><a href="#cite_ref-5">^</a> <cite id="CITEREFBelsley1991" class="citation book cs1">Belsley, David (1991). Conditioning diagnostics : collinearity and weak data in regression. New York: Wiley. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>9780471528890</bdi>.</cite>

</li>

<li id="cite_note-6"><a href="#cite_ref-6">^</a> <cite id="CITEREFBaltasDoyle2001" class="citation journal cs1">Baltas, G.; Doyle, P. (2001). "Random Utility Models in Marketing Research: A Survey". <a href="Journal_of_Business_Research" title="Journal of Business Research">Journal of Business Research</a>. 51 (2): 115–125. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1016%2FS0148-2963%2899%2900058-2">10.1016/S0148-2963(99)00058-2</a>.</cite>

</li>

<li id="cite_note-7"><a href="#cite_ref-7">^</a> <a rel="nofollow" class="external text" href="https://www.stata.com/manuals13/rmlogit.pdf">Stata Manual “mlogit — Multinomial (polytomous) logistic regression”</a>

</li>

<li id="cite_note-8"><a href="#cite_ref-8">^</a> <cite id="CITEREFDarroch,_J.N.Ratcliff,_D.1972" class="citation journal cs1">Darroch, J.N. & Ratcliff, D. (1972). <a rel="nofollow" class="external text" href="http://projecteuclid.org/download/pdf_1/euclid.aoms/1177692379">"Generalized iterative scaling for log-linear models"</a>. The Annals of Mathematical Statistics. 43 (5): 1470–1480. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1214%2Faoms%2F1177692379">10.1214/aoms/1177692379</a>.</cite>

</li>

<li id="cite_note-9"><a href="#cite_ref-9">^</a> <cite id="CITEREFBishop2006" class="citation book cs1">Bishop, Christopher M. (2006). Pattern Recognition and Machine Learning. Springer. pp. 206–209.</cite>

</li>

<li id="cite_note-10"><a href="#cite_ref-10">^</a> <cite id="CITEREFYuHuangLin2011" class="citation journal cs1">Yu, Hsiang-Fu; Huang, Fang-Lan; Lin, Chih-Jen (2011). <a rel="nofollow" class="external text" href="http://www.csie.ntu.edu.tw/~cjlin/papers/maxent_dual.pdf">"Dual coordinate descent methods for logistic regression and maximum entropy models"</a> (PDF). Machine Learning. 85 (1–2): 41–75. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10994-010-5221-8">10.1007/s10994-010-5221-8</a>.</cite>

</li>

</ol></div></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-03-03" href="https://en.wikipedia.org/wiki/?title=Multinomial_logistic_regression&oldid=1278597985">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>